{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ai2-jedi/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n",
      "Using Theano backend.\n",
      "Using gpu device 0: GeForce GTX 980 Ti (CNMeM is disabled, cuDNN Mixed dnn version. The header is from one version, but we link with a different version (4007, 5005))\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import math\n",
    "import time\n",
    "import copy\n",
    "from grid_world_v2 import GridWorld\n",
    "from replay_memory import ReplayMemory\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import model_from_json\n",
    "from keras.callbacks import Callback\n",
    "from random import shuffle\n",
    "from keras.layers import MaxPooling2D, Flatten\n",
    "from keras.layers import Convolution2D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "obstacle_grid = np.array([[0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,0,0,0,0,0,0,0],\n",
    "                          [0,0,0,0,0,0,0,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0],\n",
    "                          [0,0,0,1,1,1,1,0,0,0]])\n",
    "\n",
    "GRID_SIZE = 10\n",
    "TOTAL_CARS = 2\n",
    "TERMINAL_REWARD = 5\n",
    "DEMAND_LIMIT = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def testNaivePolicy(TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,obstacle_grid):\n",
    "    terminal = False\n",
    "    grid_world = GridWorld(num_cars=TOTAL_CARS, grid_size=GRID_SIZE, terminal_reward = TERMINAL_REWARD,\n",
    "                            demand_limit = DEMAND_LIMIT, obstacle_grid = obstacle_grid)\n",
    "\n",
    "    while(not terminal and grid_world.t < 50):\n",
    "        grid_world.t += 1\n",
    "        #grid_world.print_grid()\n",
    "        all_agents_step = grid_world.stepAll('Naive',1)\n",
    "        terminal = grid_world.isTerminal()\n",
    "        \n",
    "    return grid_world.t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testNaivePolicy(TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,obstacle_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32.558\n"
     ]
    }
   ],
   "source": [
    "moves = 0\n",
    "games = 2000\n",
    "for i in range(games):\n",
    "    print 'Game: ', i\n",
    "    moves = moves + testNaivePolicy(TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,obstacle_grid)\n",
    "    clear_output(wait=True)\n",
    "result = moves/float(games)\n",
    "print result    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Convolution2D(32, 3, 3, input_shape=(4, GRID_SIZE, GRID_SIZE),activation='relu'))\n",
    "model.add(Convolution2D(32, 3, 3, activation='relu'))\n",
    "model.add(Convolution2D(32, 3, 3,activation='relu'))\n",
    "model.add(Convolution2D(32, 3, 3,activation='relu'))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dense(5, init='lecun_uniform'))\n",
    "model.add(Activation('linear')) \n",
    "rms = RMSprop()\n",
    "model.compile(loss=\"mse\", optimizer = rms)\n",
    "\n",
    "model_json = model.to_json()\n",
    "with open(\"models/model_10x10_Conv.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load json and create model\n",
    "json_file = open('models/model_10x10_Conv.json', 'r')\n",
    "loaded_model_json = json_file.read()\n",
    "json_file.close()\n",
    "loaded_model1 = model_from_json(loaded_model_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model from disk\n"
     ]
    }
   ],
   "source": [
    "loaded_model1.load_weights(\"weights/Gridworld_6x6_4Layer_Conv_300.h5\")\n",
    "print(\"Loaded model from disk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def testModel(model, TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,obstacle_grid):\n",
    "    terminal = False\n",
    "    grid_world = GridWorld(num_cars=TOTAL_CARS, grid_size=GRID_SIZE, terminal_reward = TERMINAL_REWARD,\n",
    "                            demand_limit = DEMAND_LIMIT, obstacle_grid = obstacle_grid)\n",
    "\n",
    "    while(not terminal and grid_world.t < 50):\n",
    "        grid_world.t += 1\n",
    "        #grid_world.print_grid()\n",
    "        all_agents_step = grid_world.stepAll(model,0)\n",
    "        terminal = grid_world.isTerminal()\n",
    "        \n",
    "    return grid_world.t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    . De |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    . De |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    . De |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    . De |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    . De |\n",
      "|    .    .    .    .    .    .    .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    . C2 .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    . C2 .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . CD .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . De .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . De .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . De .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . De .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . De .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "| C1 .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . De .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "| C1 .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . De .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "| C1 .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . C2 . De .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "| C1 .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    . De .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "| C1 .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    . De .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    . De .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    . De .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    . De .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ . C2 .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    . C2 . De .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    . C2 .    |\n",
      "|    . C1 .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ . De .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ . De .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ . De .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ . De .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ . De .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    . De |\n",
      " ------------------------------------------------------\n",
      " ------------------------------------------------------\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    . C1 .    .    .    .    .    .    .    .    |\n",
      "|    .    .    .    .    .    .    .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    .    .    |\n",
      "|    .    .    . @@ . @@ . @@ . @@ .    . C2 . De |\n",
      " ------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "38"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testModel(loaded_model1,TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,obstacle_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27.7\n"
     ]
    }
   ],
   "source": [
    "moves = 0\n",
    "games = 2000\n",
    "for i in range(games):\n",
    "    print 'Game: ', i\n",
    "    moves = moves + testModel(loaded_model1, TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,obstacle_grid)\n",
    "    clear_output(wait=True)\n",
    "result = moves/float(games)\n",
    "print result    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7x7 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "obstacle_grid = np.array([[0,0,1,1,1,0,0],\n",
    "              [0,0,0,1,0,0,0],\n",
    "              [0,0,1,1,1,0,0],\n",
    "              [0,0,0,0,0,0,0],\n",
    "              [0,0,1,1,1,0,0],\n",
    "              [0,0,0,1,0,0,0],\n",
    "              [0,0,1,1,1,0,0]])\n",
    "\n",
    "GRID_SIZE = 7\n",
    "TOTAL_CARS = 2\n",
    "TERMINAL_REWARD = 5\n",
    "DEMAND_LIMIT = 1\n",
    "WAIT_TIME = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testNaivePolicy(TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,WAIT_TIME,obstacle_grid):\n",
    "    terminal = False\n",
    "    grid_world = GridWorld(num_cars=TOTAL_CARS, grid_size=GRID_SIZE, terminal_reward = TERMINAL_REWARD,\n",
    "                            demand_limit = DEMAND_LIMIT, cust_wait_time = WAIT_TIME, obstacle_grid = obstacle_grid)\n",
    "\n",
    "    while(not terminal and grid_world.t < 50):\n",
    "        grid_world.t += 1\n",
    "        grid_world.print_grid()\n",
    "        all_agents_step = grid_world.stepAll('Naive',1)\n",
    "        terminal = grid_world.isTerminal()\n",
    "        \n",
    "    return grid_world.t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      "|    .    .    . @@ .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      "|    .    .    . @@ .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      "|    .    .    . @@ . C2 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . De .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . C2 . C1 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . De .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . C2 .    .    |\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      "|    .    .    . De .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . C2 .    .    |\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      "|    .    .    . De .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . C2 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . De .    . C1 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . C2 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . De . C1 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . C2 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . C1 .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . C1 .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . C2 .    |\n",
      "|    .    .    .    . C1 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . C2 .    |\n",
      "|    .    .    .    .    . C1 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . C2 .    |\n",
      "|    .    .    . @@ .    . C1 .    |\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . De .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . De .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . C2 .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . De .    .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    . C1 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . De .    . C2 .    .    |\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . De . C2 .    . C1 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . C2 .    . C1 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    . De .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . C2 .    . C1 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    . De .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . C2 . C1 .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    . De .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    . C2 .    . C1 .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    . De .    . @@ .    .    .    |\n",
      "|    . C2 . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . C1 .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    . C2 .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . C1 .    .    .    . De |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    . C2 . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . C1 .    .    . De |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    . C2 .    .    . C1 .    . De |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . C2 .    .    . C1 . De |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testNaivePolicy(TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,WAIT_TIME,obstacle_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31.5835\n"
     ]
    }
   ],
   "source": [
    "moves = 0\n",
    "games = 2000\n",
    "for i in range(games):\n",
    "    print 'Game: ', i\n",
    "    moves = moves + testNaivePolicy(TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,WAIT_TIME,obstacle_grid)\n",
    "    clear_output(wait=True)\n",
    "result = moves/float(games)\n",
    "print result    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(256, init='lecun_uniform', input_shape=(GRID_SIZE*GRID_SIZE*4,)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(256, init='lecun_uniform'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(5, init='lecun_uniform'))\n",
    "model.add(Activation('linear')) \n",
    "rms = RMSprop()\n",
    "model.compile(loss=\"mse\", optimizer = rms)\n",
    "\n",
    "model_json = model.to_json()\n",
    "with open(\"models/model_7x7_4layer.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load json and create model\n",
    "json_file = open('models/model_7x7_4layer.json', 'r')\n",
    "loaded_model_json = json_file.read()\n",
    "json_file.close()\n",
    "loaded_model1 = model_from_json(loaded_model_json)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model from disk\n"
     ]
    }
   ],
   "source": [
    "loaded_model1.load_weights(\"weights/Gridworld_7x7_4Layer_120.h5\")\n",
    "print(\"Loaded model from disk\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testModel(model, TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,WAIT_TIME,obstacle_grid):\n",
    "    terminal = False\n",
    "    grid_world = GridWorld(num_cars=TOTAL_CARS, grid_size=GRID_SIZE, terminal_reward = TERMINAL_REWARD,\n",
    "                            demand_limit = DEMAND_LIMIT, cust_wait_time = WAIT_TIME, obstacle_grid = obstacle_grid)\n",
    "\n",
    "    while(not terminal and grid_world.t < 50):\n",
    "        grid_world.t += 1\n",
    "        grid_world.print_grid()\n",
    "        all_agents_step = grid_world.stepAll(model,0)\n",
    "        terminal = grid_world.isTerminal()\n",
    "        \n",
    "    return grid_world.t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . De .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    . C1 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . De .    .    |\n",
      "|    .    . @@ . @@ . @@ . C2 .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . C1 .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ . De . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    . C1 |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      "|    .    .    . @@ . C2 .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    . C1 |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      "|    .    .    . @@ .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    . C1 |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ . C2 .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    . C1 |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ . C2 .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    . C1 |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    . C1 |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . C2 .    |\n",
      "|    .    .    .    .    . C1 . De |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    . C2 . C1 |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    . C1 |\n",
      "|    .    .    .    .    . C2 .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    . C2 .    . C1 |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    . C1 |\n",
      "|    .    .    . C2 .    .    .    |\n",
      "| De .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    . C2 .    .    .    . C1 |\n",
      "| De .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    . C2 .    .    .    . C1 .    |\n",
      "| De .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    .    .    .    . C1 |\n",
      "| De . C2 . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n",
      " ------------------------------------\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ . De .    |\n",
      "|    .    .    .    .    . C1 .    |\n",
      "| C2 .    . @@ . @@ . @@ .    .    |\n",
      "|    .    .    . @@ .    .    .    |\n",
      "|    .    . @@ . @@ . @@ .    .    |\n",
      " ------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testModel(loaded_model1,TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,WAIT_TIME,obstacle_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24.547\n"
     ]
    }
   ],
   "source": [
    "moves = 0\n",
    "games = 2000\n",
    "for i in range(games):\n",
    "    print 'Game: ', i\n",
    "    moves = moves + testModel(loaded_model1, TOTAL_CARS,GRID_SIZE,TERMINAL_REWARD,DEMAND_LIMIT,WAIT_TIME,obstacle_grid)\n",
    "    clear_output(wait=True)\n",
    "result = moves/float(games)\n",
    "print result    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
